
sourceFile = 'usertagpreference_sorted.csv'
resultFile = 'usertagpreference_grouped.csv'
puts '[INFO] reading sorted csv file......'
begin
  inFile = File.open(sourceFile, 'r')
  outFile = File.open(resultFile, 'a')
  groupLine = []
  resultArray = []
  num = 0
  t1 = Time.now
  inFile.each_line do |line|
    curLine = line.split("\t")
    curLine[3] = curLine[3].to_i
    if curLine[0]!=groupLine[0] || curLine[1]!=groupLine[1] || curLine[2]!=groupLine[2] # found a brand new line
      if resultArray.size >= 100000
        puts '[DEBUG] writing 100000 lines into csv file in batch'
        resultArray.each do |row|
          outFile.write(row[0]<<','<<row[1]<<','<<row[2]<<','<<row[3].to_s<<"\n")
        end
        resultArray.clear
      end
      groupLine = curLine
      resultArray.push(groupLine)
    else  # found subsequent line that needs to be grouped
      groupLine[3] += curLine[3]
    end
    ## counter
    num += 1
    if num % 100000 == 0
      t2 = Time.now
      puts "[DEBUG] number of lines have processed: #{num}, average time consumption for this batch: #{t2-t1}"
      t1 = t2
    end
  end
ensure
  inFile.close
  outFile.close
end
puts '[INFO] job has done......'  

